#!/usr/bin/env python3
# -*- coding: utf-8 -*-


import re

from pyspark.sql.functions import udf
from pyspark.sql.types import ArrayType, StringType

@udf(returnType=ArrayType(StringType()))
def splitToken(cols):
    """
        1. 输入参数长度可变，字段可变
        2. 正则去除特殊字符
    """
    col=""
    for i in cols:
        if i is not None:
            col+=i

    uncn = re.compile(r'[^\u4e00-\u9fa5]')
    en = "".join(uncn.findall(col.lower()))
    b = re.sub('\W+', ' ', en).replace("_", '')
    uncz = re.compile(r'[\u4e00-\u9fa5]')
    cz = "".join(uncz.findall(col.lower()))
    d =[i for i in cz] + [i for i in b.split()]
    return d

@udf(returnType=StringType())
def sentenceClear(col):
    return re.sub('\W+', ' ', col.lower()).replace("_", '')